# -*- coding: utf-8 -*-
# article_search.py
# Created by Hardy on 2017/08/14
# Copyright 2017 杭州网川教育有限公司. All rights reserved.


import utils.utils as utils
from querier.esquerier import ElasticSearchQuerier
from utils.media_biz import media_biz_dict
from utils.wechat_article_dedup import article_deduplicate

READ = 'read_num'
LIKE = 'like_num'
RATIO = 'like_read_ratio'
RELATIVE = 'relative'

MAX_CHARACTER = 20
MINIMUM_SHOULD_MATCH = '5<85% 10<9'
MINIMUM_SHOULD_MATCH_KEYWORDS = '2<85% 3<2'
CATEGORY_CUTOFF = 0.3
MAX_KEYWORDS = 50
MIN_TEXT_LEN = 50


class ArticleSearchQuerier(ElasticSearchQuerier):
    def __init__(self, es, index, doc_type, nlp_service=None):
        super(ArticleSearchQuerier, self).__init__(es, index, doc_type)
        self.nlp_service = nlp_service

    def _build_query(self, args):
        term = args.get('term', '')
        term = term.strip() if term else ''

        filters = args.get('filters', {})
        filters = filters if filters else {}

        order = args.get('order_by', utils.ORDER_OVERALL)
        from_ = args.get('from', 0)
        size_ = args.get('size', 10)
        highlight = args.get('highlight', False)
        text_only = args.get('text_only', False)

        exact = args.get('exact', True)

        exact = True if exact is None else exact

        if len(term) > 30:
            exact = False

        deduplicate = args.get('deduplicate', True)
        if deduplicate is None:
            deduplicate = True

        img_search = args.get('image_search', False)

        filter_keywords = args.get('filter_keywords', [])
        filter_keywords = filter_keywords if filter_keywords else []

        if not filters.get('text_len') and not filters.get('biz_code'):
            filters['text_len'] = [MIN_TEXT_LEN]

        # wordcloud = args.get('wordcloud', False)

        # 处理查询文本
        keyword_type = 'keywordcloud'
        # 处理查询文本
        term2, keywords, ex_keywords, weights = utils.process_query_term(term, self.nlp_service, keyword_type,
                                                                         allowed_num_words=MAX_KEYWORDS)
        print(keywords)

        # delay = datetime.timedelta(DAYS)
        # today = datetime.datetime.today()
        # prev_date = today - delay
        #
        # today = str(today)[0:19]
        # prev_date = str(prev_date)[0:19]

        # if 'publish_timestamp' not in filters:
        #     filters['publish_timestamp'] = [prev_date, today]

        ex_category = []

        ex_kw = utils.get_kv_json(ex_keywords, weights)
        # ex_category = self.nlp_service.classify(keywords)
        # ex_category = ex_category.get('classify', {})
        # ex_category = utils.get_kv_json(ex_category.get('category', []), ex_category.get('category_prob', []))
        # keywords = keywords * 2 + ex_keywords

        h_kv = {}

        for i in range(0, len(keywords)):
            if h_kv.get(keywords[i]) is None:
                h_kv[keywords[i]] = i

        if exact:
            # term, query_keywords, filter_words, filters, order, from_, size_, highlight):
            query = self._gen_query_mini(term, ' '.join(keywords[0:10]), filter_keywords, filters, order, from_, size_,
                                         highlight)
        else:
            query = self._gen_query_mx(term, ' '.join(keywords[0:50]), filter_keywords, filters, order, from_, size_,
                                       highlight, text_only=text_only)

        keywords = [k[0] for k in sorted(h_kv.items(), key=lambda x: x[1])]
        return query, {}, {'keywords': keywords, 'order': order, 'ex_keywords': ex_kw, "ex_category": ex_category,
                           'deduplicate': deduplicate, 'highlight': highlight, 'filters': filters,
                           'image_search': img_search}

    def _build_result(self, es_result, param):
        keywords = param['keywords']
        ex_keywords = param['ex_keywords']
        ex_category = param['ex_category']
        filters = param['filters']
        img_search = param['image_search']
        # order = param['order']

        deduplicate = param['deduplicate']
        total = es_result['hits']['total']
        articles = []
        for hit in es_result['hits']['hits']:
            articles.append(self.extract_result(hit, param['highlight'], img_search))

        if deduplicate and not filters.get('biz_code'):
            try:
                articles = article_deduplicate(articles, key='title_simhash', remove=False)
                # articles = article_deduplicate(articles, key='text_simhash', remove=False)

            except Exception as e:
                print(str(e))

        return {
            'total': total,
            'keywords': keywords,
            'ex_keywords': ex_keywords,
            'ex_category': ex_category,
            'articles': articles
        }

    def _gen_query_mx(self, term, query_keywords, filter_words, filters, order, from_, size_,
                      highlight, text_only):
        must_clause = []
        should_clause = []
        filter_clause = []
        filter_clause = self.get_filters(filter_clause, filters)

        if 0 < len(term) <= 5:
            should_clause.append(
                {
                    'match_phrase': {
                        "title": {
                            'query': term,
                            'slop': 1,
                            'boost': 1,
                        },
                    },
                }
            )

        if query_keywords.strip():
            must_clause.append(
                {
                    'multi_match': {
                        'analyzer': 'whitespace',
                        'query': query_keywords,
                        'fields': ['title_seg^2', 'keywords'] if not text_only else ['keywords'],
                        'boost': 1
                        # 'minimum_should_match': ""
                    }
                }
            )

        if filter_words is not None:
            filter_clause = self._add_filter_clause_search_out_of_filter(filter_clause, filter_words)
        # for words in filter_words:
        #     filter_clause = self.filter_keywords(filter_clause, words)

        query = {"query": {
            "bool": {
                "filter": filter_clause,
            }
        }, 'from': from_, 'size': size_}

        if must_clause:
            query['query']['bool']['must'] = must_clause

        if should_clause:
            query['query']['bool']['should'] = should_clause
            # query['query']['bool']['minimum_should_match'] = 1

        query = self._get_sort(query, order, term, filters)

        query['track_scores'] = True

        if highlight:
            query['highlight'] = {
                "pre_tags": ["<span class='keyword'>"],
                "post_tags": ["</span>"],
                "fields": {"keywords": {}, "title": {}, 'title_seg': {}}
            }
        else:
            query['highlight'] = {
                "pre_tags": [""],
                "post_tags": [""],
                "fields": {"keywords": {}, "title": {}, 'title_seg': {}}
            }

        return query

    def _gen_query_mini(self, term, query_keywords, filter_words, filters, order, from_, size_, highlight):
        must_clause = []
        should_clause = []
        filter_clause = []

        filter_clause = self.get_filters(filter_clause, filters)

        if query_keywords.strip():
            should_clause.append(
                {
                    'multi_match': {
                        'analyzer': 'whitespace',
                        'query': query_keywords,
                        'fields': ['title_seg'],
                        'boost': 1,
                        # 'minimum_should_match': MINIMUM_SHOULD_MATCH_KEYWORDS
                    }
                }
            )

        if filter_words is not None:
            filter_clause = self._add_filter_clause_search_out_of_filter(filter_clause, filter_words)
        # for words in filter_words:
        #     filter_clause = self.filter_keywords(filter_clause, words)

        if term:
            term = term.strip()
            if 0 < len(term) <= 10:
                should_clause.append(
                    {
                        'match_phrase': {
                            "title": {
                                'query': term[0:MAX_CHARACTER],
                                'slop': 2,
                                'boost': 3,
                            },
                        },
                    }
                )
            else:
                should_clause.append(
                    {
                        'match': {
                            "title": {
                                'query': term[0:MAX_CHARACTER],
                                'boost': 30,
                                'minimum_should_match': MINIMUM_SHOULD_MATCH
                            },

                        }
                    }
                )

        query = {"query": {
            "bool": {
                "filter": filter_clause,
            }
        }, 'from': from_, 'size': size_}

        if must_clause:
            query['query']['bool']['must'] = must_clause

        if should_clause:
            query['query']['bool']['should'] = should_clause
            query['query']['bool']['minimum_should_match'] = 1

        query = self._get_sort(query, order, term, filters)

        query['track_scores'] = True
        if highlight:
            query['highlight'] = {
                "pre_tags": ["<span class='keyword'>"],
                "post_tags": ["</span>"],
                "fields": {"title": {"number_of_fragments": 2}, 'title_seg': {}}
            }

        return query

    @staticmethod
    def _get_sort(query, order, term, filters):
        if order == 'read_num':
            query['sort'] = [
                # '_score',
                {'read_num': 'desc'}
            ]
        elif order == 'like_num':
            query['sort'] = [
                # '_score',
                {'like_num': 'desc'}
            ]
        elif order == 'like_read_ratio':
            query['sort'] = [
                # '_score',
                {'like_read_ratio': 'desc'}
            ]
        elif order == 'relative':
            pass
        else:
            query['sort'] = [
                {
                    '_script': {
                        "type": "number",
                        "script": {
                            "lang": "painless",
                            "inline": "_score * Math.log(doc.publish_date.value + 1.1) "
                                      "* Math.log(doc.category_weight.value + 1.0) "
                                      "* Math.log(doc.read_num.value + 1000.0)"
                        },
                        "order": "desc",
                    },
                },
                '_score'
            ]
            if not term.strip():
                query['sort'] = [{'publish_timestamp': 'desc'}]

            if filters.get('category'):
                query['sort'] = [{'category_weight': 'desc'}] + query['sort']

            if filters.get('category_social'):
                query['sort'] = [{'category_social_weight': 'desc'}] + query['sort']

        return query

    @staticmethod
    def filter_festivals(must_clause, festival_keywords):
        clause = []
        must_clause.append({
            'bool': {'should': clause}
        })
        for fk in festival_keywords:
            q = {
                'match_phrase': {
                    "title": {
                        'query': fk,
                        'slop': 1,
                    },
                }}
            clause.append(q)

        q_keywords = {
            'multi_match': {
                'analyzer': 'whitespace',
                'query': ' '.join(festival_keywords),
                'fields': ['keywords', 'title_seg^3'],
                # 'minimum_should_match': ""
            }
        }
        clause.append(q_keywords)
        return must_clause

    @staticmethod
    def filter_keywords(must_clause, keywords):
        clause = []
        must_clause.append({
            'bool': {'should': clause, 'minimum_should_match': 1}
        })
        for fk in keywords:
            q = {
                'match_phrase': {
                    "title": {
                        'query': fk,
                        'slop': 1,
                    },
                }}
            clause.append(q)
        return must_clause

    @staticmethod
    def _add_filter_clause_search_out_of_filter(filter_clause, keywords, key='text'):
        clause = []
        filter_clause.append({
            'bool': {
                'should': clause,
                'minimum_should_match': 1
            }
        })
        for keyword in keywords:
            clause.append({'match_phrase': {key: {"query": keyword}}})
        return filter_clause

    @staticmethod
    def _add_filter_clause_search(filter_clause, keywords_list, key='text'):
        clause = []
        filter_clause.append({
            'bool': {
                'should': clause,
                'minimum_should_match': 1
            }
        })

        for keywords in keywords_list:
            must_clause = []
            for keyword in keywords:
                must_clause.append({'match_phrase': {key: {"query": keyword}}})
            clause.append({'bool': {'must': must_clause}})
        return filter_clause

    @staticmethod
    def _add_filter_match(must_clause, filters, key, cond='must'):
        if key in filters:
            if filters[key]:
                clause = []
                must_clause.append({
                    'bool': {cond: clause}
                })
                values = filters[key]
                if isinstance(values, str):
                    values = values.split(' ')
                for fk in values:
                    clause.append({'match': {key: {'query': fk, 'minimum_should_match': MINIMUM_SHOULD_MATCH}}})
        return must_clause

    @staticmethod
    def _add_filter_match_biz(must_clause, filters, key):
        if key in filters:
            if filters[key]:
                clause = []
                must_clause.append({
                    'bool': {'must': clause}
                })
                values = filters[key]
                if isinstance(values, str):
                    values = values.split(' ')
                clause.append({'terms': {key: values}})
                # for fk in values:
                #     clause.append({'match': {key: {'query': fk, 'minimum_should_match': MINIMUM_SHOULD_MATCH}}})
        return must_clause

    @staticmethod
    def _add_filter_match_phrase(must_clause, filters, key, cond='must'):
        if key in filters:
            if filters[key]:
                clause = []
                must_clause.append({
                    'bool': {cond: clause}
                })
                values = filters[key]
                if isinstance(values, str):
                    values = values.split(' ')
                for fk in values:
                    q = {
                        'match_phrase': {
                            "title": {
                                'query': fk,
                                'slop': 1,
                            },
                        }}

                    clause.append(q)
        return must_clause

    @staticmethod
    def _add_filter_clause(filter_clause, filters, key, cond='must'):
        if key in filters:
            if filters[key]:
                clause = []
                filter_clause.append({
                    'bool': {
                        cond: clause
                    }
                })
                for fk in filters[key]:
                    clause.append({'term': {key: fk}})
        return filter_clause

    @staticmethod
    def _add_filter_clause_keyword(filter_clause, filters, key, cond='must'):
        if key in filters:
            if filters[key]:
                clause = []
                filter_clause.append({
                    'bool': {
                        cond: clause
                    }
                })
                fks = filters[key]
                clause.append({'multi_match': {'query': ' '.join(fks), 'analyzer': 'whitespace', 'fields': [key]}})
        return filter_clause

    @staticmethod
    def _add_filter_range_clause(filter_clause, filters, key):
        if key in filters:
            if filters[key]:
                clause = []
                filter_clause.append({
                    'bool': {
                        'must': clause
                    }
                })
                fk = filters[key]
                if not isinstance(fk, list) or len(fk) < 1:
                    pass
                else:
                    min_fk = fk[0]
                    if len(fk) >= 2:
                        max_fk = fk[1]
                    else:
                        max_fk = None
                    if min_fk is not None and min_fk != 'null' and min_fk != '':
                        clause.append({'range': {key: {"gte": min_fk}}})
                    if max_fk is not None and max_fk != 'null' and max_fk != '':
                        clause.append({'range': {key: {"lte": max_fk}}})
        return filter_clause

    def get_filters(self, filter_clause, filters):
        if filters:
            filter_clause = self._add_filter_clause(filter_clause, filters, 'from')
            filter_clause = self._add_filter_clause(filter_clause, filters, 'source')
            filter_clause = self._add_sentiment_filter_clause(filter_clause, filters, 'full_sentiment_category')
            filter_clause = self._add_filter_clause(filter_clause, filters, 'has_copyright', 'should')
            filter_clause = self._add_filter_range_clause(filter_clause, filters, 'publish_timestamp')
            filter_clause = self._add_filter_range_clause(filter_clause, filters, 'read_num')
            filter_clause = self._add_filter_range_clause(filter_clause, filters, 'like_num')
            filter_clause = self._add_filter_range_clause(filter_clause, filters, 'like_read_ratio')
            filter_clause = self._add_filter_range_clause(filter_clause, filters, 'image_num')
            filter_clause = self._add_filter_range_clause(filter_clause, filters, 'video_num')
            filter_clause = self._add_filter_range_clause(filter_clause, filters, 'text_len')

            cm = filters.get('category_media')
            if cm:
                filters['biz_code'] = media_biz_dict[cm[0]]

            filter_clause = self._add_filter_match_biz(filter_clause, filters, 'biz_code')

            if filters.get('category'):
                filters['category'] = [utils.category_smzdm_2_encode(c) for c in filters['category']]
                if not filters.get('category_weight'):
                    filters['category_weight'] = [CATEGORY_CUTOFF]
                filter_clause = self._add_filter_clause(filter_clause, filters, 'category', 'should')
                filter_clause = self._add_filter_range_clause(filter_clause, filters, 'category_weight')

            if filters.get('category_social'):
                # filters['category_social'] = [utils.category_social_encode(c) for c in filters['category_social']]
                if not filters.get('category_social_weight'):
                    filters['category_social_weight'] = [CATEGORY_CUTOFF]
                filter_clause = self._add_filter_clause_keyword(filter_clause, filters, 'category_social')
                filter_clause = self._add_filter_range_clause(filter_clause, filters, 'category_social_weight')

            if filters.get('category_taobao'):
                filter_clause = self._add_filter_clause_keyword(filter_clause, filters, 'category_taobao')

            title_filter_keywords = filters.get('title_filter_keywords')
            if title_filter_keywords:
                filter_clause = self._add_filter_clause_search(filter_clause, title_filter_keywords, 'title')

            text_filter_keywords = filters.get('text_filter_keywords')
            if text_filter_keywords:
                filter_clause = self._add_filter_clause_search(filter_clause, text_filter_keywords, 'text')

        return filter_clause

    @staticmethod
    def _add_sentiment_filter_clause(filter_clause, filters, key):
        if key in filters:
            if filters[key] == [1]:
                clause = []
                filter_clause.append({
                    'bool': {
                        'should': clause
                    }
                })
                for fk in [1, 2]:
                    clause.append({'term': {key: fk}})
            elif filters[key] == [5]:
                clause = []
                filter_clause.append({
                    'bool': {
                        'should': clause
                    }
                })
                for fk in [4, 5]:
                    clause.append({'term': {key: fk}})
            else:
                clause = []
                filter_clause.append({
                    'bool': {
                        'must': clause
                    }
                })
                for fk in filters[key]:
                    clause.append({'term': {key: fk}})

        return filter_clause

    @staticmethod
    def extract_result(hit, is_highlight, image_search):
        source_ = dict(hit['_source'])
        url = source_.get('url')
        likes = source_.get('like_num', 0)
        reads = source_.get('read_num', 0)

        keywords = source_['keywords']

        highlight = hit.get('highlight', {})
        h_keywords = []

        h_title = source_.get('title', '')
        from_ = source_.get('from')
        if str(from_) == '29' or str(from_) == '30':
            url = 'https://market.m.taobao.com/apps/market/content/index.html?contentId=%s' % source_.get('id')

        # h_title_seg = source_['title_seg']

        if is_highlight:
            # h_title = # utils.highlight_text(h_title, keyword_params)
            # if h_title == source_['title'] or str(from_) == '9':
            h_title = highlight.get('title')
            if h_title:
                h_title = h_title[0]
            else:
                h_title = source_['title']

        h_keywords = h_keywords if h_keywords else keywords[0:10]

        h_kv = {}

        for i in range(0, len(h_keywords)):
            if h_kv.get(h_keywords[i]) is None:
                h_kv[h_keywords[i]] = i

        h_keywords = [k[0] for k in sorted(h_kv.items(), key=lambda x: x[1])]

        if from_ == 1:
            reads = reads if reads < 100000 else 100001
            likes = likes if likes < 100000 else 100001
        msg_cdn_url = source_.get('msg_cdn_url', '')
        res = {
            'id': source_.get('id'),
            'article_id': source_.get('article_id'),
            'biz_code': source_.get('biz_code', ''),
            'biz_name': source_.get('biz_name', ''),
            'title': utils.clean_text(h_title),
            'title_simhash': source_.get('title_simhash'),
            'text_simhash': source_.get('text_simhash'),
            'raw_keywords': source_.get('keywords'),
            'url': url,
            'qrcode': utils.get_qrcode(url) if str(source_.get('from')) == '1' else '',
            'msg_cdn_url': msg_cdn_url if msg_cdn_url[0:2] != '//' else 'http:' + msg_cdn_url,
            'keywords': h_keywords,
            'read_num': reads,
            'like_num': likes,
            'has_copyright': source_.get('has_copyright'),
            'publish_timestamp': source_.get('publish_timestamp'),
            'crawler_timestamp': source_.get('crawler_timestamp'),
            'category_weight': source_.get('category_weight'),
            'category': utils.category_smzdm_2_decode(source_.get('category', -1)),
            'category_social': utils.category_social_decode(source_.get('category_social', -1)),
            'image_num': source_.get('image_num'),
            'video_num': source_.get('video_num'),
            'text_len': source_.get('text_len'),
            'abstract': source_.get('abstract'),
            'from': source_.get('from'),
            'sentiment': source_.get('full_sentiment_category', ''),
            'sentiment_prob': source_.get('sentiment_category_weight', ''),
            'brands': []  # self.nlp_service.get_brands(source_.get('keywords', []))
        }
        if image_search:
            res['image_list'] = get_img_url(source_.get('image_list', []), source_['from'])

        return res


def get_img_url(img_list, from_):
    img_list_out = []
    if img_list is not None:
        if str(from_) == '1':
            for i in range(len(img_list)):
                if img_list[i][0:4] != 'http' and len(img_list[i]) > 4:
                    img_list_out.append('http://mmbiz.qpic.cn/' + img_list[i])
        else:
            for i in range(len(img_list)):
                if img_list[i][0:4] != 'http':
                    img_list_out.append('http://' + img_list[i])
                else:
                    img_list_out.append(img_list[i])
    else:
        img_list_out = []
    return img_list_out
